#!/usr/bin/php
<?php
#######################################################################################################
#Title: Yahoo! Pipe definition retriever 
#Description: This script retrieves the (Yahoo!) pipes in JSON format
#Author: Carlos Rodriguez
#Creation Date: 09.06.2011
#Last update  : 10.06.2011
#######################################################################################################

include 'util.php';

$matched = array();

if($argc < 5 || $argv[1] != "-f" || $argv[3] != "-d"){
	echo "Usage: pipe_crawler -f file_path_with_list_of_pipes_ids -d target_directory\n";
	exit;
}

$file_path = $argv[2];
$target_dir = $argv[4];

if(!file_exists($file_path)){
	echo "Error: file does not exists.\n";
	exit;
}

$lines = file($file_path);

foreach($lines as $line){
	#echo $line;
	$line = trim($line);
	if($line == '' || $line[0] == '#'){
		continue;
	}

	$url = "http://query.yahooapis.com/v1/public/yql?q=select%20PIPE.working%20from" . 
		"%20json%20where%20url%3D%22http%3A%2F%2Fpipes.yahoo.com%2Fpipes%2Fpipe.info" . 
		"%3F_out%3Djson%26_id%3D" . $line . "%22&format=json";

	$page = get_web_page($url);	
	$content =  $page['content'];

	$filename = $target_dir . "pipe_" . $line;

	$fh = fopen($filename, 'w') or die('Cannot create file in target directory');
	fwrite($fh, $content);
	fclose($fh);

	#sleep(1);

}

?>
